{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32cda5b5-6855-430b-befb-a7a62e70218a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "f57a1845-fff7-4983-af95-e73e4b403014",
   "metadata": {},
   "source": [
    "## 本地数据库方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfa16915-9b41-4b03-a6ad-99d238065dd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain_community.embeddings import ZhipuAIEmbeddings\n",
    "from langchain_milvus import Milvus\n",
    "\n",
    "# 文件导入\n",
    "loader = WebBaseLoader(\"https://zh.d2l.ai/\")\n",
    "docs = loader.load()\n",
    "\n",
    "# 文本切分\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=1500,\n",
    "    chunk_overlap=150\n",
    ")\n",
    "splits = text_splitter.split_documents(docs)\n",
    "print(len(splits))\n",
    "\n",
    "# 文本嵌入\n",
    "embed = ZhipuAIEmbeddings(model=\"embedding-2\", api_key=\"5713143e8fdc4b4a8b284cf97092e70f.qEK71mGIlavzO1Io\")\n",
    "\n",
    "# 向量库创建 - 添加 index_params to specify supported index type\n",
    "vectorstore = Milvus.from_documents(\n",
    "    documents=splits,  # You should use the splits, not the original docs\n",
    "    embedding=embed,\n",
    "    connection_args={\n",
    "        \"uri\": \"./milvus_demo.db\",\n",
    "    },\n",
    "    index_params={\n",
    "        \"metric_type\": \"L2\",  # or \"IP\" depending on your needs\n",
    "        \"index_type\": \"IVF_FLAT\",  # Use a supported index type\n",
    "        \"params\": {\"nlist\": 128}  # IVF_FLAT parameter\n",
    "    },\n",
    "    drop_old=True,\n",
    ")\n",
    "\n",
    "\n",
    "# 检索\n",
    "question = \"图像识别\"\n",
    "docs = vectorstore.similarity_search(question,k=3)\n",
    "print(len(docs))\n",
    "print(docs[0].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8985d5b7-b2f5-421b-820b-9332d8a8358b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "782d636b-4ccc-402e-9eab-4154a98e67c1",
   "metadata": {},
   "source": [
    "## 远端数据库方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "086c1380-01ac-460c-b05c-cdd83ac30bba",
   "metadata": {},
   "outputs": [],
   "source": [
    "!docker port milvus-standalone 19530/tcp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b00941f8-3703-46fc-bcdc-319b1d71ad49",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57a35e8d-7e81-40bd-ab5d-7f0be2c8db73",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import connections, db\n",
    " \n",
    "conn = connections.connect(host=\"129.201.70.31\", port=19530)\n",
    "database = db.create_database(\"sample_db\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "279e3dd0-861a-4029-85ee-6fd480cc895c",
   "metadata": {},
   "outputs": [],
   "source": [
    "测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8668a2d8-f8ab-4627-abcc-ed83993931fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import (\n",
    "    db,\n",
    "    MilvusClient,\n",
    "    FieldSchema, CollectionSchema, DataType,\n",
    "    Collection,\n",
    ")\n",
    "\n",
    "#1.创建Milvus客户端\n",
    "fmt = \"\\n=== {:30} ===\\n\"\n",
    "# 1. connect to Milvus 数据库必须选存在，可通过可视化的管理界面创建数据库\n",
    "print(fmt.format(\"1. start connecting to Milvus\"))\n",
    "milvusclient = MilvusClient(uri=\"http://120.79.252.32:19530\", db_name=\"default\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "536c735a-9be0-4713-9bb6-404779e7d991",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import MilvusClient\n",
    "\n",
    "client = MilvusClient(\n",
    "    uri=\"http://localhost:19530\"\n",
    ")\n",
    "\n",
    "client.list_databases()\n",
    "\n",
    "\n",
    "# client.describe_database(\n",
    "#     db_name=\"default\"\n",
    "# )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1519840-fbce-483f-a02e-06870adee106",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15613258-1703-46fe-8a8e-28c249f292e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import bs4\n",
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "loader = WebBaseLoader(\n",
    "    web_paths=(\n",
    "        \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
    "        \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
    "    ),\n",
    "    bs_kwargs=dict(\n",
    "        parse_only=bs4.SoupStrainer(\n",
    "            class_=(\"post-content\", \"post-title\", \"post-header\")\n",
    "        )\n",
    "    ),\n",
    ")\n",
    "documents = loader.load()\n",
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=200)\n",
    "\n",
    "docs = text_splitter.split_documents(documents)\n",
    "\n",
    "docs[1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "875bfc2b-a6a5-44ed-bd4b-fd25adbbee15",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.vectorstores import Milvus\n",
    "from langchain_community.embeddings import ZhipuAIEmbeddings\n",
    "\n",
    "embed = ZhipuAIEmbeddings(model=\"embedding-2\",api_key=\"5713143e8fdc4b4a8b284cf97092e70f.qEK71mGIlavzO1Io\")\n",
    "vector = Milvus.from_documents(\n",
    "     documents=documents, # 设置保存的文档\n",
    "     embedding=embed, # 设置 embedding model\n",
    "     collection_name=\"book2\", # 设置 集合名称\n",
    "     drop_old=True,\n",
    "     connection_args={\"host\": \"120.79.252.32\", \"port\": \"19530\", \"db_name\":\"sample_db\"},# Milvus连接配置\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31a3e0bf-986b-4042-9b4f-462c1b89e78e",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"What is self-reflection of an AI Agent?\"\n",
    "vector.similarity_search(query, k=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c09ff928-5c8c-4f50-a1cc-8bb62a9c962d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain_community.embeddings import ZhipuAIEmbeddings\n",
    "from langchain_milvus import Milvus\n",
    "# 文件导入\n",
    "loader = WebBaseLoader(\"https://zh.d2l.ai/\")\n",
    "docs = loader.load()\n",
    "\n",
    "# 文本切分\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size = 1500,\n",
    "    chunk_overlap = 150\n",
    ")\n",
    "splits = text_splitter.split_documents(docs)\n",
    "print(len(splits))\n",
    "\n",
    "# 文本嵌入\n",
    "embed = ZhipuAIEmbeddings(model=\"embedding-2\",api_key=\"5713143e8fdc4b4a8b284cf97092e70f.qEK71mGIlavzO1Io\")\n",
    "\n",
    "# 测试\n",
    "# text_1 = \"今天天气不错\"\n",
    "\n",
    "# query_result = embeddings.embed_query(text_1)\n",
    "# print(query_result)\n",
    "\n",
    "# 路径设置\n",
    "# persist_directory = './langchain-milvus'\n",
    "\n",
    "# 向量库创建\n",
    "connection_args = {\n",
    "    \"host\": \"129.201.70.31\",\n",
    "    \"port\": \"19530\",\n",
    "}\n",
    "vectordb = Milvus.from_documents(\n",
    "    documents=docs,\n",
    "    embedding=embed,\n",
    "    collection_name=\"book1\",\n",
    "    drop_old=False,\n",
    "    connection_args=connection_args,\n",
    ")\n",
    "# print(vectordb._collection.count())\n",
    "\n",
    "# 检索\n",
    "question = \"图像识别\"\n",
    "docs = vectordb.similarity_search(question,k=3)\n",
    "print(len(docs))\n",
    "print(docs[0].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68da3144-ed85-4daa-90af-d2021dedc027",
   "metadata": {},
   "outputs": [],
   "source": [
    "# "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env_rag",
   "language": "python",
   "name": "env_rag"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
